const puppeteer = require('puppeteer-extra');
const stealthPlugin = require('puppeteer-extra-plugin-stealth');
const cheerio  = require('cheerio');
const path = require("node:path");
const fs = require("node:fs");
const XLSX= require("xlsx");
const {create} = require("axios");
puppeteer.use(stealthPlugin());

const config = {
    targetUrl: process.argv[2] || 'https://qmed.com/components-code005395.html',
    outputPath: process.argv[3] || path.join(__dirname, 'companies.xlsx'),
};

async function browserContent(link) {
    const browser = await puppeteer.launch({
        headless: true,
        args: [
            '--no-sandbox',
            '--disable-setuid-sandbox',
            '--disable-web-security',
            '--disable-features=IsolateOrigins,site-per-process'
        ]
    });
    const page = await browser.newPage();
    await page.setRequestInterception(true);
    page.on('request', (request) => {
        const allowedTypes = ['document'];
        if (allowedTypes.includes(request.resourceType())) {
            request.continue();
        } else {
            request.abort();
        }
    });
    await page.goto(link, { waitUntil: 'domcontentloaded' } );
    const content= await page.content();
    if (!content) {
        throw new Error('No content');
        return Error;
    }
    //console.log(typeof(content));
    await browser.close();

    return content;
}


async function crawl(mainpage) {
    const page1 = await browserContent(mainpage).then(result => {
        return result;
    });
    console.log("正在整理网页...");

    const $ = cheerio.load(page1);

    const targetLinks = [];

    $("#companies .card").each((i, el) => {
        const links = {
            link: $(el).find('a').attr('href'),
        }
        targetLinks.push(links);
    })
    console.log(targetLinks);
    return targetLinks;

}


async function infoSniffer(index,links) {

    const targetInfo = [];
    for (const link of links) {

        const pages = await browserContent(index + link.link).then(result => {
            return result;
        });

        const $ = cheerio.load(pages);

        $(".widget:first ul").each((i, el) => {
            const name = $(el).find('strong').text().trim().replace(",", '');
            const oneinfo = [
                name,
                $(el).find('li').text().toString().replace(",", ' ').replace("  ", " ").replace(name, ''),
            ]
            console.log(oneinfo);
            targetInfo.push(oneinfo);
        })


    }
    return targetInfo;
}

async function output(content, filepath) {
    const worksheet = XLSX.utils.aoa_to_sheet(content);

    const workbook = XLSX.utils.book_new();

    XLSX.utils.book_append_sheet(workbook,worksheet,'company info');

    const dir = path.dirname(filepath);
    if (!fs.existsSync(dir)) {
        fs.mkdirSync(dir, { recursive: true });
    }

    XLSX.writeFile(workbook, filepath);
    console.log("文件已保存!");

}




(async () => {

    const index = config.targetUrl.replace(/^((https?:\/\/[^/]+)\/?).*/, '$2');
    console.log("正在解析地址...");
    const links = await crawl(config.targetUrl).then(result => {
        return result;
    });

    console.log(links);
    console.log("正在获取信息...")

    const infos = await infoSniffer(index,links).then(result => {
        return result;
    });

    console.log("信息获取完毕，正在生成...");
    output(infos, config.outputPath);

    return 0;

})();